# -*- coding: utf-8 -*-

# @Project : fastapi-tutorial
# @Date    : 20240325-0017
# @Author  : robin

import os
import sys
from dotenv import load_dotenv
# Load environment variables from openai.env file
load_dotenv("../.env")

# Loader机制

# 加载markdown
# 加载cvs
# 加载文件目录
# 加载html
# 加载JSON
# 加载PDF

# from langchain.document_loaders import TextLoader
#
# loader = TextLoader("loader.md", encoding='utf-8')
# data = loader.load()
# print(data)
#
# #使用loader来加载cvs文件
# from langchain.document_loaders.csv_loader import CSVLoader
#
# #loader = CSVLoader(file_path="loader.csv")
# loader = CSVLoader(file_path="loader.csv", encoding='utf-8', source_column="Location")
# data = loader.load()
# print(data)

#! pip install "unstructured[xlsx]"
from langchain.document_loaders import DirectoryLoader

#目录下的.html和.rst文件不会被这种loader加载
#loader = DirectoryLoader("目录地址",glob="指定加载说明格式的文件")
# loader = DirectoryLoader(path="./example/",glob="*.xlsx")
# docs = loader.load()
# # len(docs)
# print(len(docs))
# print(docs)

#使用loader来加载html文件
#from langchain.document_loaders import UnstructuredHTMLLoader

#loader = UnstructuredHTMLLoader("loader.html")
# from langchain.document_loaders import BSHTMLLoader
# from langchain_community.document_loaders.html_bs import BSHTMLLoader
#
# loader = BSHTMLLoader("loader.html", open_encoding="utf-8")
# data = loader.load()
# print(data)

# ! pip intsall jq
#使用loader来加载json文件
#需要先安装 ! pip install jq
#

# from langchain.document_loaders import JSONLoader
# loader = JSONLoader(
#     file_path = "simple_prompt.json",jq_schema=".template",text_content=True
# )
# data = loader.load()
# print(data)

# ! pip install pypdf

#loader加载pdf文件
from langchain_community.document_loaders import PyPDFLoader
# pip install pypdf2
# 如果需要让langchain 可以调用pdfloader 必须要先使用传统的pdf 可以跑通
import PyPDF2

pdf_file    = open("loader.pdf", "rb")
pdf_reader  = PyPDF2.PdfReader(pdf_file)

print(len( pdf_reader.pages ))
page_one = pdf_reader.pages[0]
print(page_one.extract_text())

from langchain_community.document_loaders.pdf import PyPDFLoader
loader = PyPDFLoader("loader.pdf")
pages = loader.load_and_split()
print(pages[0].page_content)

# from pypdf import PdfReader
# reader = PdfReader("loader.pdf")
# number_of_pages = len(reader.pages)
# page = reader.pages[0]
# text = page.extract_text().encode("utf-8")
# print(text)

